#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import requests
import re

def crawl_python100examples(base_url):
    '''
    利用requests库请求获取网页源代码，再利用BeautifulSoup解析网页
    '''
    #base_url = 'http://www.runoob.com/python/python-exercise-example2.html'
    #请求头（固定格式）
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36','referer':'http://www.runoob.com/python/python-100-examples.html'}
    #执行GET请求
    html = requests.get(base_url,headers=headers)
    #对网页中汉字进行转码，其中utf-8是根据网页源代码中设置的编码格式来指定的
    html.encoding='utf-8'
    content = html.text
    #利用BeautifulSoup进行解析
    soup = BeautifulSoup(content,'lxml')
    l=[]
    #re过滤\n\r\t条件
    pattern = re.compile(r'\n|\r|\t')
    #查找所有P标签（tag）
    for i in soup.find_all('p'):
        #筛选指定tag内容
        if re.match('^程序分析|题目',i.text):
            #利用re过滤\n\r\t
            s=re.sub(pattern,'',i.text)
            l.append(s)
            l.append('\n')
    return l

explain = "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#Python 100例\n\n"

#数据持久化，保存到本地文件中，注意字符转码encoding=UTF-8
with open(r'D:\myGit\Python100examples.py','w',encoding='UTF-8') as f:
    f.write(explain)
    for n in range(1,101):
        base_url = 'http://www.runoob.com/python/python-exercise-example'+str(n)+'.html'
        s = crawl_python100examples(base_url)
        f.write('\'\'\'\n')
        f.write('No.'+str(n)+'\n')
        f.writelines(s)
        f.write('\'\'\'\n')
        f.write('\n'*5)

print('Complete!')





















